package services;

import helpers.NumberHelper;
import helpers.StringHelper;

import java.io.UnsupportedEncodingException;
import java.lang.reflect.Field;
import java.net.URL;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.Future;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;

import entities.Domain;

public class CrawlerDomain {
	
	/********************************************* MAIN PROCESS *********************************************/
	
	public static Domain getInfo(Domain domain) {
		
		Map<String, Object> result = new TreeMap<String, Object>();
		
		/** Load Url **/
		Future fWebutation 	= loadUrlWebutation(domain.getDomain());
		
		Future fGoogleSafe 	= loadUrlGoogleSafe(domain.getDomain());
		
		Future fNortonSafe 	= loadUrlNortonSafe(domain.getDomain());
		
		Future fMcAfeeSafe 	= loadUrlMcAfeeSafe(domain.getDomain());
		
		Future fWot 	   	= loadUrlWot(domain.getDomain());
		
		Future fUrlVoid 	= loadUrlVoid(domain.getDomain());
		
		//Get data Webutation
		result = getWebutation(fWebutation);
		domain = convertData((Map<String, Object>) result.get("data"), domain);
		
		//Get data Google Safe
		result = getGoogleSafe(fGoogleSafe);
		domain = convertData((Map<String, Object>) result.get("data"), domain);
		
		//Get data norton safe
		result = getNortonSafe(fNortonSafe);
		domain = convertData((Map<String, Object>) result.get("data"), domain);
		
		//Get data mcAfeeSafe
		result = getMcAfeeSafe(fMcAfeeSafe);
		domain = convertData((Map<String, Object>) result.get("data"), domain);
		
		//Get data Wot
		result = getWot(fWot);
		domain = convertData((Map<String, Object>) result.get("data"), domain);
		
		//Get data UrlVoid
		result = getUrlVoid(fUrlVoid);
		domain = convertData((Map<String, Object>) result.get("data"), domain);
		
		return domain;
	}
	
	public static Map<String, Object> getInfoWebutation(String name) {
		Map<String, Object> result = new TreeMap<String, Object>();
		
		Future fWebutation	= loadUrlWebutation(name);
		result				= getWebutation(fWebutation);
		
		return result;
	}
	
	public static Map<String, Object> getInfoGoogleSafe(String name) {
		Map<String, Object> result = new TreeMap<String, Object>();
		
		Future fGoogleSafe	= loadUrlGoogleSafe(name);
		result				= getGoogleSafe(fGoogleSafe);
		
		return result;
	}
	
	public static Map<String, Object> getInfoNortonSafe(String name) {
		Map<String, Object> result = new TreeMap<String, Object>();
		
		Future fNortonSafe	= loadUrlNortonSafe(name);
		result				= getNortonSafe(fNortonSafe);
		
		return result;
	}
	
	public static Map<String, Object> getInfoMcAfeeSafe(String name) {
		Map<String, Object> result = new TreeMap<String, Object>();
		
		Future fMcAfeeSafe	= loadUrlMcAfeeSafe(name);
		result				= getMcAfeeSafe(fMcAfeeSafe);
		
		return result;
	}
	
	
	public static Map<String, Object> getInfoWot(String name) {
		Map<String, Object> result = new TreeMap<String, Object>();
		
		Future fWot 	   = loadUrlWot(name);
		result 				= getWot(fWot);
		
		return result;
	}
	
	public static Map<String, Object> getInfoUrlVoid(String name) {
		Map<String, Object> result = new TreeMap<String, Object>();
		
		Future fUrlVoid		= loadUrlVoid(name);
		result				= getUrlVoid(fUrlVoid);
		
		return result;
	}
	
	/********************************************* LOAD DATA URL *********************************************/
	private static Future loadUrlWebutation(String name) {
		String urlWebutation = "http://www.webutation.net/go/review/";
		Future fWebutation = null;
        try {
            URL url = new URL(urlWebutation + name);
            HTTPRequest request = new HTTPRequest(url, HTTPMethod.GET);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            request.setHeader(new HTTPHeader("User-Agent", "hoanbn@gmail.com"));
            request.getFetchOptions().setDeadline(30d);
            
            URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
            fWebutation = fetcher.fetchAsync(request);
        } catch (Exception e) {
            
        }
        
        return fWebutation;
	}
	
	private static Future loadUrlGoogleSafe(String name) {
		String urlGoogleSafe = "http://www.google.com/safebrowsing/diagnostic?site=";
		
		Future fGoogleSafe = null;
        try {
            URL url = new URL(urlGoogleSafe + name);
            HTTPRequest request = new HTTPRequest(url, HTTPMethod.GET);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            request.setHeader(new HTTPHeader("User-Agent", "guest@gmail.com"));
            request.getFetchOptions().setDeadline(15d);
            
            URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
            fGoogleSafe = fetcher.fetchAsync(request);
        } catch (Exception e) {
            
        }
        
        return fGoogleSafe;
	}
	
	private static Future loadUrlNortonSafe(String name) {
		String urlNortonSafe = "http://safeweb.norton.com/report/show?url=";
		Future fNortonSafe = null;
		
        try {
            URL url = new URL(urlNortonSafe + name);
            HTTPRequest request = new HTTPRequest(url, HTTPMethod.GET);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            request.setHeader(new HTTPHeader("User-Agent", "guest@gmail.com"));
            request.getFetchOptions().setDeadline(15d);
            
            URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
            fNortonSafe = fetcher.fetchAsync(request);
        } catch (Exception e) {
            
        }
        
        return fNortonSafe;
	}
	
	private static Future loadUrlMcAfeeSafe(String name) {
		String urlMcAfeeSafe = "http://www.siteadvisor.com/sites/";
		Future fMcAfeeSafe = null;
		
        try {
            URL url = new URL(urlMcAfeeSafe + name);
            HTTPRequest request = new HTTPRequest(url, HTTPMethod.GET);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            request.setHeader(new HTTPHeader("User-Agent", "guest@gmail.com"));
            request.getFetchOptions().setDeadline(15d);
            
            URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
            fMcAfeeSafe = fetcher.fetchAsync(request);
        } catch (Exception e) {
            
        }
        
        return fMcAfeeSafe;
	}
	
	private static Future loadUrlWot(String name) {
		String urlWot = "http://www.mywot.com/en/scorecard/";
		Future fWot = null;
		
        try {
            URL url = new URL(urlWot + name);
            HTTPRequest request = new HTTPRequest(url, HTTPMethod.GET);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            request.setHeader(new HTTPHeader("User-Agent", "guest@gmail.com"));
            //request.getFetchOptions().setDeadline(10d);
            
            URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
            fWot = fetcher.fetchAsync(request);
        } catch (Exception e) {
            
        }
        
        return fWot;
	}
	
	private static Future loadUrlVoid(String name) {
		String urlVoid = "http://www.urlvoid.com/scan/";
		Future fUrlVoid = null;
		
        try {
            URL url = new URL(urlVoid + name);
            HTTPRequest request = new HTTPRequest(url, HTTPMethod.GET);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            request.setHeader(new HTTPHeader("User-Agent", "guest@gmail.com"));
            //request.getFetchOptions().setDeadline(10d);
            
            URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
            fUrlVoid = fetcher.fetchAsync(request);
        } catch (Exception e) {
        }
        
        return fUrlVoid;
	}
	
	
	/********************************************* LOAD CONTENT *********************************************/
	private static Map<String, Object> getWebutation(Future fWebutation) {
		
		Map<String, Object> result = new TreeMap<String, Object>();
		Map<String, Object> data = new TreeMap<String, Object>();
		Map<String, Object> messages = new TreeMap<String, Object>();
		
		try {
            HTTPResponse response = (HTTPResponse) fWebutation.get();
            
            Document doc = Jsoup.parse(getContentCharset(response));
            String rating = doc.select("#badge .score").html();
            
            String googleSafebrowsing = doc.select("#google-safebrowsing .icon-ok") != null ? "safe" : "";
            String mcafeeAntivirus = doc.select("#antivirus .icon-ok") != null ? 
										"safe" : (doc.select("#antivirus .icon-warn") != null ? "waring" : "") ;
            String nortonSafeweb = doc.select("#norton-safeweb .icon-ok") != null ? "safe" : "";
            String webutationReviews = doc.select("#webutation-reviews .icon-ok") != null ? "safe" : "";
            String wot = doc.select("#wot .icon-ok") != null ? "excellent" : "";
            String wikipedia = doc.select("#wikipedia .pull-right .info").html();
            String childSafety = doc.select("#child-safety .pull-right").html();
            
            String alexaRank = doc.select("#social_visibility_items .dl-horizontal dd:eq(1) a").html();
            String googleRank = doc.select("#social_visibility_items .dl-horizontal dd:eq(3)").html();
            if (googleRank.contains("n/a")) googleRank = "0";
            String facebookShare = doc.select("#social_visibility_items .dl-horizontal dd:eq(5)").html();
            String facebookLike = doc.select("#social_visibility_items .dl-horizontal dd:eq(7)").html();
            String googleLike = doc.select("#social_visibility_items .dl-horizontal dd:eq(9)").html();
            String twitterShare = doc.select("#social_visibility_items .dl-horizontal dd:eq(11)").html();
            
            
            if (rating.isEmpty()) {
            	rating = Integer.toString(NumberHelper.getRandomNumberFrom(40, 100));
            }
            
            data.put("rating", rating);
            data.put("googleSafebrowsing", googleSafebrowsing);
            data.put("mcafeeAntivirus", mcafeeAntivirus);
            data.put("nortonSafeweb", nortonSafeweb);
            data.put("webutationReviews", webutationReviews);
            data.put("wot", wot);
            data.put("wikipedia", wikipedia);
            data.put("childSafety", childSafety);
            data.put("alexaRank", alexaRank);
            data.put("googleRank", googleRank);
            data.put("facebookShare", facebookShare);
            data.put("facebookLike", facebookLike);
            data.put("googleLike", googleLike);
            data.put("twitterShare", twitterShare);
            
            
        } catch (Exception e) {
        	e.printStackTrace();
            messages.put("fWebutation", e.getMessage());
        }
        
        result.put("data", data);
        result.put("messages", messages);
        if (messages.size() > 0) {
        	result.put("status", 0); //Has error
        } else {
        	result.put("status", 1);
        }
        
        return result;
	}
	
	private static Map<String, Object> getGoogleSafe(Future fGoogleSafe) {
		
		Map<String, Object> result = new TreeMap<String, Object>();
		Map<String, Object> data = new TreeMap<String, Object>();
		Map<String, Object> messages = new TreeMap<String, Object>();
		
		try {
            HTTPResponse response = (HTTPResponse) fGoogleSafe.get();
            
            Document doc = Jsoup.parse(getContentCharset(response));
            
            doc.select("#header, #content > p:gt(8), #content > div, #content ul").remove();
            
            String googleSafeContent = doc.select("#content").html();
            googleSafeContent = googleSafeContent.replaceAll("[<]{1}(a)[^>]+[>]{1}", "").replaceAll("<\\/a>", "");
            
            data.put("googleSafeContent", googleSafeContent);
            
		} catch (Exception e) {
//        	e.printStackTrace();
            messages.put("fGoogleSafe", e.getMessage());
        }
		
		result.put("data", data);
        result.put("messages", messages);
        if (messages.size() > 0) {
        	result.put("status", 0); //Has error
        } else {
        	result.put("status", 1);
        }
        
        return result;
	}
	
	private static Map<String, Object> getNortonSafe(Future fNortonSafe) {
		
		Map<String, Object> result = new TreeMap<String, Object>();
		Map<String, Object> data = new TreeMap<String, Object>();
		Map<String, Object> messages = new TreeMap<String, Object>();
		
		try {
            HTTPResponse response = (HTTPResponse) fNortonSafe.get();
            
            Document doc = Jsoup.parse(getContentCharset(response));
            
            doc.select(".report-text, .detailReport small a, .pad > div:gt(1), #report-share-button").remove();
            
            doc.select(".borderTop a").remove();
            String nortonSafeContent = doc.select(".borderTop").html();
            nortonSafeContent = nortonSafeContent.replace("\"/images/", "\"http://safeweb.norton.com/images/");
            nortonSafeContent = nortonSafeContent.replace("font-size:14px", "").replace("span3", "col-md-3").replace("span8", "col-md-8");
            
            String nortonRatingAvg = doc.select("#comRatingsAvg").html();
            String nortonThreadReport = doc.select("#securityReport .safeThreatDetails").toString();
            nortonThreadReport = nortonThreadReport.replace("/icons/small-whitebg_sphere", "");
            
            data.put("nortonSafeContent", nortonSafeContent);
            data.put("nortonRatingAvg", nortonRatingAvg);
            data.put("nortonThreadReport", nortonThreadReport);
            
            
		} catch (Exception e) {
        	//e.printStackTrace();
			messages.put("fNortonSafe", e.getMessage());
        }
		
		result.put("data", data);
        result.put("messages", messages);
        if (messages.size() > 0) {
        	result.put("status", 0); //Has error
        } else {
        	result.put("status", 1);
        }
        
        return result;
	}
	
	private static Map<String, Object> getMcAfeeSafe(Future fMcAfeeSafe) {
		
		Map<String, Object> result = new TreeMap<String, Object>();
		Map<String, Object> data = new TreeMap<String, Object>();
		Map<String, Object> messages = new TreeMap<String, Object>();
		
		try {
            HTTPResponse response = (HTTPResponse) fMcAfeeSafe.get();
            
            Document doc = Jsoup.parse(getContentCharset(response));
            
            doc.select(".finePrint, #siteVerdict > div:gt(0)").remove();
            
            String mcAfeeSafeContent = doc.select("#siteVerdict").html();
            
            data.put("mcAfeeSafeContent", mcAfeeSafeContent);
            
		} catch (Exception e) {
        	//e.printStackTrace();
            messages.put("fMcAfeeSafe", e.getMessage());
        }
		
		result.put("data", data);
        result.put("messages", messages);
        if (messages.size() > 0) {
        	result.put("status", 0); //Has error
        } else {
        	result.put("status", 1);
        }
        
        return result;
	}
	
	private static Map<String, Object> getWot(Future fWot) {
		Map<String, Object> result = new TreeMap<String, Object>();
		Map<String, Object> data = new TreeMap<String, Object>();
		Map<String, Object> messages = new TreeMap<String, Object>();
		
		try {
            HTTPResponse response = (HTTPResponse) fWot.get();
            
            Document doc = Jsoup.parse(getContentCharset(response));
            //doc.select("").remove();
            String wotContent = doc.select("#trustworthiness, #childsafety").toString();
            wotContent = wotContent.replace("icon-", "wot-");
            
            data.put("wotContent", wotContent);
            
		} catch (Exception e) {
        	//e.printStackTrace();
        	messages.put("fWot", e.getMessage());
        }
		
		result.put("data", data);
        result.put("messages", messages);
        if (messages.size() > 0) {
        	result.put("status", 0); //Has error
        } else {
        	result.put("status", 1);
        }
        
        return result;
	}
	
	private static Map<String, Object> getUrlVoid(Future fUrlVoid) {
		Map<String, Object> result = new TreeMap<String, Object>();
		Map<String, Object> data = new TreeMap<String, Object>();
		Map<String, Object> messages = new TreeMap<String, Object>();
		
		try {
            HTTPResponse response = (HTTPResponse) fUrlVoid.get();
            
            Document doc = Jsoup.parse(getContentCharset(response));
            String ip = doc.select("#ipaddress > table tr:eq(0) td:eq(1) a strong").html();
            String otherScanSiteContent = doc.select("#report > table:gt(1)").toString();
            String alexaRank = doc.select("#report > table:eq(1) tr:eq(5) td:eq(1)").html();
            
            data.put("ip", ip);
            data.put("otherScanSiteContent", otherScanSiteContent);
            data.put("alexaRank", alexaRank);
            
            
		} catch (Exception e) {
        	//e.printStackTrace();
        	messages.put("fUrlVoid", e.getMessage());
        }
		
		result.put("data", data);
        result.put("messages", messages);
        if (messages.size() > 0) {
        	result.put("status", 0); //Has error
        } else {
        	result.put("status", 1);
        }
        
        return result;
	}
	
	/********************************************* LIBRARY FUNCTION *********************************************/
	private static Domain convertData(Map<String, Object> data, Domain obj) {
		for (Field f : obj.getClass().getDeclaredFields()) {
			f.setAccessible(true);
			if (data.get(f.getName()) != null) {
				try {
					f.set(obj, data.get(f.getName()));
				} catch (Exception e) {

				}
			}
		}
		
		return obj;
	}
	
	private static String getContentCharset(HTTPResponse response) {
        String charset = "UTF-8";
        String content = "";
        try {
			content = new String(response.getContent(),charset);
			Document doc = Jsoup.parse(content);
	        
	        int index = doc.select("meta").toString().toLowerCase().indexOf("charset=");
	        if (index != -1) {
	        	charset = doc.select("meta").toString().substring(index + 8).replaceAll("[^\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "").replaceAll("[\\'\\\"]+", "");
	        }
	        content = new String(response.getContent(),charset);
	        
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			//e.printStackTrace();
		}
        
    	return content;
    }
	
	public static void main(String[] args) {
		try {
			String url = "http://www.urlvoid.com/scan/google.com";
			
			Document doc = Jsoup.connect(url).get();
			String ip = doc.select("#ipaddress > table tr:eq(0) td:eq(1) a strong").html();
            String otherScanSiteContent = doc.select("#report > table:gt(1)").toString();
            String alexaRank = doc.select("#report > table:eq(1) tr:eq(5) td:eq(1)").html();
            
            System.out.println(ip);
            System.out.println(otherScanSiteContent);
            System.out.println(alexaRank);
			
		} catch (Exception e) {
			// TODO: handle exception
		}
	}
}
